# -*- coding: utf-8 -*-
import scrapy
from qiubai import db
import sys
from scrapy import Request
import time
import json
class ArticleSpider(scrapy.Spider):
    name = "article"
    allowed_domains = ["qiushibaike.com"]
    num = 5
    #count = 0
    start_urls = ( 'http://www.qiushibaike.com/',)


    def start_requests(self):
        headers={"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36"}
        lastId = db.getLastId()
        endId = lastId+self.num
        db.addLastId(endId)
        preUrl = "http://m2.qiushibaike.com/article/"
        endUrl = "/comments?article=1"
        i = 0
        for i in range(1,self.num):
            curId = lastId+i
            url = preUrl+str(curId)+endUrl
            print url;
            yield Request(url=url,headers=headers)
            #print "sleeping................."
            time.sleep(3);

    def parse(self, response):
        content = response.body
        jsonObj = json.loads(content)
        if(jsonObj['err'] !=0):
            return
        article = jsonObj['article']
        spider_id = article['id']
        format = article['format']
        content = article['content']
        image = article['image']
        image_size = json.dumps(article['image_size'])
        if "video" == format:
            pic_url = article['pic_url']
            low_url = article['low_url']
            high_url = article['high_url']
        else:
            pic_url = "";
            low_url = "";
            high_url = "";

        user = json.dumps(article['user'])
        db.insertQiubai(spider_id,format,content,image,image_size,pic_url,low_url,high_url,user);
        
